{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = 'C:\\\\Users\\\\bai\\\\Downloads\\\\Train\\\\Train_DataSet.csv'\n",
    "label_data = 'C:\\\\Users\\\\bai\\\\Downloads\\\\Train\\\\Train_DataSet_Label.csv'\n",
    "train_df = pd.read_csv(train_data)\n",
    "label_df = pd.read_csv(label_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7a3dd79f90ee419da87190cff60f7a86</td>\n",
       "      <td>问责领导(上黄镇党委书记张涛，宣国才真能一手遮天吗？)</td>\n",
       "      <td>这几天看了有人举报施某某的贴子，经与举报人联系证实，是宣某当天中午请举报人和枪手喝酒后，晚上...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7640a5589bc7486ca199eeeb38af79dd</td>\n",
       "      <td>江歌事件:教会孩子，善良的同时更要懂得保护自己!</td>\n",
       "      <td>过去一年的江歌悲剧，这几日再次刷屏：住在东京都中野区的中国女留学生江歌，收留了被前男友陈世锋...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8c5bda93e4ba401f90a0faa5b28fe57f</td>\n",
       "      <td>绝味鸭脖广告\"开黄腔\"引众怒 \"双11\"这么拼值吗?</td>\n",
       "      <td>“双11”1600亿的销售额让中国乃至全世界感到震惊，为此，不仅有不惜欠债百万的“剁手党”，...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1aa777fed31a4b8a9d866f05b5477557</td>\n",
       "      <td>央视曝光!如东一医药企业将槽罐车改成垃圾车，夜间偷排高浓度废水</td>\n",
       "      <td>2016年以来，如东高新区一家叫做拜瑞生物医药的企业竟然将装废水的槽罐车伪装成洒水车，常年在...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6c67ac55360340258e157f3710ebae6c</td>\n",
       "      <td>恶劣至极，央视都曝光了!南通如东一医药企业将槽罐车改成洒水车，夜间偷排高浓度废水...丢大发了!</td>\n",
       "      <td>新闻资讯•民生热点•城市生活•南通人都在看浏览更多资讯请关注微信公众号:南通传媒网昨晚央视一...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 id  \\\n",
       "0  7a3dd79f90ee419da87190cff60f7a86   \n",
       "1  7640a5589bc7486ca199eeeb38af79dd   \n",
       "2  8c5bda93e4ba401f90a0faa5b28fe57f   \n",
       "3  1aa777fed31a4b8a9d866f05b5477557   \n",
       "4  6c67ac55360340258e157f3710ebae6c   \n",
       "\n",
       "                                              title  \\\n",
       "0                       问责领导(上黄镇党委书记张涛，宣国才真能一手遮天吗？)   \n",
       "1                          江歌事件:教会孩子，善良的同时更要懂得保护自己!   \n",
       "2                        绝味鸭脖广告\"开黄腔\"引众怒 \"双11\"这么拼值吗?   \n",
       "3                   央视曝光!如东一医药企业将槽罐车改成垃圾车，夜间偷排高浓度废水   \n",
       "4  恶劣至极，央视都曝光了!南通如东一医药企业将槽罐车改成洒水车，夜间偷排高浓度废水...丢大发了!   \n",
       "\n",
       "                                             content  \n",
       "0  这几天看了有人举报施某某的贴子，经与举报人联系证实，是宣某当天中午请举报人和枪手喝酒后，晚上...  \n",
       "1  过去一年的江歌悲剧，这几日再次刷屏：住在东京都中野区的中国女留学生江歌，收留了被前男友陈世锋...  \n",
       "2  “双11”1600亿的销售额让中国乃至全世界感到震惊，为此，不仅有不惜欠债百万的“剁手党”，...  \n",
       "3  2016年以来，如东高新区一家叫做拜瑞生物医药的企业竟然将装废水的槽罐车伪装成洒水车，常年在...  \n",
       "4  新闻资讯•民生热点•城市生活•南通人都在看浏览更多资讯请关注微信公众号:南通传媒网昨晚央视一...  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7a3dd79f90ee419da87190cff60f7a86</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7640a5589bc7486ca199eeeb38af79dd</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8c5bda93e4ba401f90a0faa5b28fe57f</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1aa777fed31a4b8a9d866f05b5477557</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6c67ac55360340258e157f3710ebae6c</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 id  label\n",
       "0  7a3dd79f90ee419da87190cff60f7a86      2\n",
       "1  7640a5589bc7486ca199eeeb38af79dd      1\n",
       "2  8c5bda93e4ba401f90a0faa5b28fe57f      2\n",
       "3  1aa777fed31a4b8a9d866f05b5477557      2\n",
       "4  6c67ac55360340258e157f3710ebae6c      2"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x25cd04f46d8>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD4CAYAAADlwTGnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAENVJREFUeJzt3X+s3fVdx/Hnay2gcYsUuSO1rZZozWQmdngtJPsHx4TC/igmLoEYaQixmpS4RWPsTAxukwQTlWTJJHahW2d0SKaGZqtiZVvMYoBeZu3oKnJlOK5t4GoZShbRsrd/3E+zQ7k/zr29Pafs83wkJ+f7fX8+3/N9f3PDffX741xSVUiS+vOWcTcgSRoPA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUqbXjbmAxl19+eW3evHncbUjSm8qTTz75H1U1sdS8CzoANm/ezNTU1LjbkKQ3lST/Nsw8LwFJUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOnVBfxFs1Dbv+fy4Wzivnrv3feNuQdIFxDMASeqUASBJnTIAJKlTBoAkdcoAkKROLRkASb4nyRNJ/inJsSQfbvVPJfl6kiPttbXVk+RjSaaTHE1y9cBn7UzyTHvtPH+HJUlayjCPgb4KvKeqXklyEfDlJH/dxn6jqj571vybgC3tdQ1wP3BNksuAu4FJoIAnkxyoqpdW40AkScuz5BlAzXmlrV7UXrXIJjuAT7ftHgMuTbIeuBE4VFWn2i/9Q8D2c2tfkrRSQ90DSLImyRHgReZ+iT/ehu5pl3nuS3JJq20Anh/YfKbVFqqfva9dSaaSTM3Ozi7zcCRJwxoqAKrqtaraCmwEtiX5CeBDwDuAnwYuA36zTc98H7FI/ex97a2qyaqanJhY8n9pKUlaoWU9BVRV3wS+BGyvqpPtMs+rwCeBbW3aDLBpYLONwIlF6pKkMRjmKaCJJJe25e8F3gv8c7uuT5IAtwBPtU0OALe3p4GuBV6uqpPAI8ANSdYlWQfc0GqSpDEY5img9cD+JGuYC4yHqupzSb6QZIK5SztHgF9p8w8CNwPTwLeAOwCq6lSSjwKH27yPVNWp1TsUSdJyLBkAVXUUeNc89fcsML+A3QuM7QP2LbNHSdJ54DeBJalTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4tGQBJvifJE0n+KcmxJB9u9SuTPJ7kmSR/nuTiVr+krU+38c0Dn/WhVn86yY3n66AkSUsb5gzgVeA9VfWTwFZge5Jrgd8D7quqLcBLwJ1t/p3AS1X1o8B9bR5JrgJuBd4JbAf+KMma1TwYSdLwlgyAmvNKW72ovQp4D/DZVt8P3NKWd7R12vj1SdLqD1bVq1X1dWAa2LYqRyFJWrah7gEkWZPkCPAicAj4V+CbVXW6TZkBNrTlDcDzAG38ZeAHBuvzbCNJGrGhAqCqXquqrcBG5v7V/uPzTWvvWWBsofrrJNmVZCrJ1Ozs7DDtSZJWYFlPAVXVN4EvAdcClyZZ24Y2Aifa8gywCaCNfz9warA+zzaD+9hbVZNVNTkxMbGc9iRJyzDMU0ATSS5ty98LvBc4DnwR+Pk2bSfwcFs+0NZp41+oqmr1W9tTQlcCW4AnVutAJEnLs3bpKawH9rcndt4CPFRVn0vyNeDBJL8L/CPwQJv/APAnSaaZ+5f/rQBVdSzJQ8DXgNPA7qp6bXUPRz3bvOfz427hvHru3veNuwV9l1kyAKrqKPCueerPMs9TPFX1P8D7F/ise4B7lt+mJGm1+U1gSeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1askASLIpyReTHE9yLMkHWv13kvx7kiPtdfPANh9KMp3k6SQ3DtS3t9p0kj3n55AkScNYO8Sc08CvV9VXkrwNeDLJoTZ2X1X9/uDkJFcBtwLvBH4Q+LskP9aGPw78LDADHE5yoKq+thoHIklaniUDoKpOAifb8n8nOQ5sWGSTHcCDVfUq8PUk08C2NjZdVc8CJHmwzTUAJGkMlnUPIMlm4F3A4610V5KjSfYlWddqG4DnBzababWF6mfvY1eSqSRTs7Ozy2lPkrQMQwdAkrcCfwF8sKr+C7gf+BFgK3NnCH9wZuo8m9ci9dcXqvZW1WRVTU5MTAzbniRpmYa5B0CSi5j75f+nVfWXAFX1wsD4J4DPtdUZYNPA5huBE215obokacSGeQoowAPA8ar6w4H6+oFpPwc81ZYPALcmuSTJlcAW4AngMLAlyZVJLmbuRvGB1TkMSdJyDXMG8G7gF4GvJjnSar8F3JZkK3OXcZ4Dfhmgqo4leYi5m7ungd1V9RpAkruAR4A1wL6qOraKxyJJWoZhngL6MvNfvz+4yDb3APfMUz+42HaSpNHxm8CS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSp5YMgCSbknwxyfEkx5J8oNUvS3IoyTPtfV2rJ8nHkkwnOZrk6oHP2tnmP5Nk5/k7LEnSUoY5AzgN/HpV/ThwLbA7yVXAHuDRqtoCPNrWAW4CtrTXLuB+mAsM4G7gGmAbcPeZ0JAkjd6SAVBVJ6vqK235v4HjwAZgB7C/TdsP3NKWdwCfrjmPAZcmWQ/cCByqqlNV9RJwCNi+qkcjSRrasu4BJNkMvAt4HLiiqk7CXEgAb2/TNgDPD2w202oL1c/ex64kU0mmZmdnl9OeJGkZhg6AJG8F/gL4YFX912JT56nVIvXXF6r2VtVkVU1OTEwM254kaZmGCoAkFzH3y/9Pq+ovW/mFdmmH9v5iq88AmwY23wicWKQuSRqDYZ4CCvAAcLyq/nBg6ABw5kmencDDA/Xb29NA1wIvt0tEjwA3JFnXbv7e0GqSpDFYO8ScdwO/CHw1yZFW+y3gXuChJHcC3wDe38YOAjcD08C3gDsAqupUko8Ch9u8j1TVqVU5CknSsi0ZAFX1Zea/fg9w/TzzC9i9wGftA/Ytp0FJ0vnhN4ElqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkTi0ZAEn2JXkxyVMDtd9J8u9JjrTXzQNjH0oyneTpJDcO1Le32nSSPat/KJKk5RjmDOBTwPZ56vdV1db2OgiQ5CrgVuCdbZs/SrImyRrg48BNwFXAbW2uJGlM1i41oar+PsnmIT9vB/BgVb0KfD3JNLCtjU1X1bMASR5sc7+27I4lSaviXO4B3JXkaLtEtK7VNgDPD8yZabWF6pKkMVlpANwP/AiwFTgJ/EGrZ565tUj9DZLsSjKVZGp2dnaF7UmSlrKiAKiqF6rqtar6NvAJvnOZZwbYNDB1I3Bikfp8n723qiaranJiYmIl7UmShrCiAEiyfmD154AzTwgdAG5NckmSK4EtwBPAYWBLkiuTXMzcjeIDK29bknSulrwJnOQzwHXA5UlmgLuB65JsZe4yznPALwNU1bEkDzF3c/c0sLuqXmufcxfwCLAG2FdVx1b9aCRJQxvmKaDb5ik/sMj8e4B75qkfBA4uqztJ0nnjN4ElqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktSpJQMgyb4kLyZ5aqB2WZJDSZ5p7+taPUk+lmQ6ydEkVw9ss7PNfybJzvNzOJKkYQ1zBvApYPtZtT3Ao1W1BXi0rQPcBGxpr13A/TAXGMDdwDXANuDuM6EhSRqPJQOgqv4eOHVWeQewvy3vB24ZqH+65jwGXJpkPXAjcKiqTlXVS8Ah3hgqkqQRWuk9gCuq6iRAe397q28Anh+YN9NqC9XfIMmuJFNJpmZnZ1fYniRpKat9Ezjz1GqR+huLVXurarKqJicmJla1OUnSd6w0AF5ol3Zo7y+2+gywaWDeRuDEInVJ0pisNAAOAGee5NkJPDxQv709DXQt8HK7RPQIcEOSde3m7w2tJkkak7VLTUjyGeA64PIkM8w9zXMv8FCSO4FvAO9v0w8CNwPTwLeAOwCq6lSSjwKH27yPVNXZN5YlSSO0ZABU1W0LDF0/z9wCdi/wOfuAfcvqTpJ03vhNYEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOrXk9wAkaRQ27/n8uFs4b567933jbmFengFIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdOqcASPJckq8mOZJkqtUuS3IoyTPtfV2rJ8nHkkwnOZrk6tU4AEnSyqzGGcDPVNXWqpps63uAR6tqC/BoWwe4CdjSXruA+1dh35KkFTofl4B2APvb8n7gloH6p2vOY8ClSdafh/1LkoZwrgFQwN8meTLJrla7oqpOArT3t7f6BuD5gW1nWu11kuxKMpVkanZ29hzbkyQt5Fz/fwDvrqoTSd4OHEryz4vMzTy1ekOhai+wF2BycvIN45Kk1XFOZwBVdaK9vwj8FbANeOHMpZ32/mKbPgNsGth8I3DiXPYvSVq5FQdAku9L8rYzy8ANwFPAAWBnm7YTeLgtHwBub08DXQu8fOZSkSRp9M7lEtAVwF8lOfM5f1ZVf5PkMPBQkjuBbwDvb/MPAjcD08C3gDvOYd+SpHO04gCoqmeBn5yn/p/A9fPUC9i90v1JklaX3wSWpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOjXyAEiyPcnTSaaT7Bn1/iVJc0YaAEnWAB8HbgKuAm5LctUoe5AkzRn1GcA2YLqqnq2q/wUeBHaMuAdJErB2xPvbADw/sD4DXDM4IckuYFdbfSXJ0yPqbRwuB/5jVDvL741qT93w5/fm9d3+s/vhYSaNOgAyT61et1K1F9g7mnbGK8lUVU2Ouw+tjD+/Ny9/dnNGfQloBtg0sL4RODHiHiRJjD4ADgNbklyZ5GLgVuDAiHuQJDHiS0BVdTrJXcAjwBpgX1UdG2UPF5guLnV9F/Pn9+blzw5IVS09S5L0XcdvAktSpwwASeqUASBJnRr19wAkaeSSvIO5vzqwgbnvHp0ADlTV8bE2NmaeAUhDSvKOJNcneetZ9e3j6klLS/KbzP3ZmQBPMPc4eoDP9P4HKX0K6AKQ5I6q+uS4+9DCkvwqsBs4DmwFPlBVD7exr1TV1ePsTwtL8i/AO6vq/86qXwwcq6ot4+ls/DwDuDB8eNwNaEm/BPxUVd0CXAf8dpIPtLH5/sSJLhzfBn5wnvr6NtYt7wGMSJKjCw0BV4yyF63Imqp6BaCqnktyHfDZJD+MAXCh+yDwaJJn+M4fo/wh4EeBu8bW1QXAS0AjkuQF4EbgpbOHgH+oqvn+haILRJIvAL9WVUcGamuBfcAvVNWasTWnJSV5C3N/jn4Dc//NzQCHq+q1sTY2Zp4BjM7ngLcO/gI5I8mXRt+Olul24PRgoapOA7cn+ePxtKRhVdW3gcfG3ceFxjMASeqUN4ElqVMGgCR1ygCQpE4ZAJLUqf8H1r+EOz6fNgAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "label_df['label'].value_counts().plot.bar()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "content_length_count = {0:0}\n",
    "for index, row in train_df.iterrows():\n",
    "    c_len = 0\n",
    "    if type(row['content']) != str:\n",
    "        content_length_count[0] += 1\n",
    "        continue\n",
    "    for x in row['content']:\n",
    "        c_len += 1\n",
    "    c_len = int(c_len / 100) + 1\n",
    "    if c_len not in content_length_count:\n",
    "        content_length_count[c_len] = 1\n",
    "    else:\n",
    "        content_length_count[c_len] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAD6xJREFUeJzt3W2MXFd9x/HvrzEJJIg6DwYF2+oaYfFQVJrISg1UCMVUkAThvCBSKgQWdeU3KQSCRJzyAvVdUBEBJJrKigFTRTzUpI1FUtrICar6Ii42oJBg0rghjU0MWUQSEIiCxb8v5ljZbtbe2eysZ+fs9yOt5t5zz8z8z9zd3949c+duqgpJUr9+b9wFSJKWlkEvSZ0z6CWpcwa9JHXOoJekzhn0ktQ5g16SOmfQS1LnDHpJ6tyqcRcAcNFFF9XU1NS4y5CkiXLo0KGfVtWa+foti6Cfmpri4MGD4y5DkiZKkv8Zpp9TN5LUOYNekjpn0EtS5wx6SeqcQS9JnTPoJalzBr0kdc6gl6TOGfSS1DmD/hSmdt7F1M67xl2GJC2aQS9JnTPoJalzBr0kdc6gl6TOGfSS1DmDXpI6Z9BLUucMeknqnEEvSZ0z6CWpcwa9JHXOoJekzhn0ktQ5g16SOmfQS1LnDHpJ6pxBL0mdGyrok3woyUNJHkzypSQvTLIhyYEkjyT5SpKzW99z2vqRtn1qKQcgSTq9eYM+yVrgA8CmqnodcBZwLfBx4Jaq2gg8BWxvd9kOPFVVrwRuaf0kSWMy7NTNKuBFSVYB5wLHgcuBvW37HuDqtry1rdO2b0mS0ZQrSVqoeYO+qn4EfAJ4nEHAPwMcAp6uqhOt2zFgbVteCxxt9z3R+l84+3GT7EhyMMnB6enpxY5DknQKw0zdnM/gKH0D8HLgPOCKObrWybucZtuzDVW7qmpTVW1as2bN8BVLkhZkmKmbtwI/rKrpqvotcAfwRmB1m8oBWAc80ZaPAesB2vbfB3420qolSUMbJugfBzYnObfNtW8Bvg/cB7yr9dkG3NmW97V12vZ7q+o5R/SSpDNjmDn6AwzeVP028L12n13AjcANSY4wmIPf3e6yG7iwtd8A7FyCuiVJQ1o1fxeoqo8BH5vV/Chw2Rx9fw1cs/jSJEmj4CdjJalzBr0kdc6gl6TOGfSS1DmDXpI6Z9BLUucMeknqnEEvSZ0z6CWpcwa9JHXOoJekzhn0ktQ5g16SOmfQS1LnDHpJ6pxBL0mdM+glqXMGvSR1zqCXpM4Z9JLUOYNekjpn0EtS5wx6SeqcQS9JnTPoJalzBr0kdc6gl6TOGfSS1DmDXpI6Z9BLUucMeknqnEEvSZ0z6CWpcwa9JHXOoJekzhn0ktQ5g16SOjdU0CdZnWRvkh8kOZzkDUkuSHJPkkfa7fmtb5J8JsmRJA8kuXRphyBJOp1hj+g/DXyjql4NvB44DOwE9lfVRmB/Wwe4AtjYvnYAt460YknSgswb9EleArwZ2A1QVb+pqqeBrcCe1m0PcHVb3gp8sQbuB1YnuXjklUuShjLMEf0rgGng80m+k+S2JOcBL6uq4wDt9qWt/1rg6Iz7H2ttkqQxGCboVwGXArdW1SXAL3l2mmYumaOtntMp2ZHkYJKD09PTQxUrSVq4YYL+GHCsqg609b0Mgv8nJ6dk2u2TM/qvn3H/dcATsx+0qnZV1aaq2rRmzZrnW78kaR7zBn1V/Rg4muRVrWkL8H1gH7CttW0D7mzL+4D3trNvNgPPnJzikSSdeauG7Pd+4PYkZwOPAu9j8Eviq0m2A48D17S+dwNXAkeAX7W+kqQxGSroq+q7wKY5Nm2Zo28B1y2yLknSiPjJWEnqnEEvSZ0z6CWpcwa9JHXOoJekzhn0ktQ5g16SOmfQS1LnDHpJ6pxBL0mdM+glqXMGvSR1zqCXpM4Z9JLUOYNekjpn0EtS5wx6SeqcQS9JnTPoJalzBr0kdc6gl6TOGfSS1DmDXpI6Z9BLUucM+iUwtfMupnbeNe4yJAkw6CWpewb9GHnkL+lMMOglqXMG/RLzqF3SuBn0ktQ5g16SOrdq3AWsRE7lSDqTPKKXpM4Z9JLUuRUT9J79ImmlWjFBL0kr1YoMeo/uJa0kQwd9krOSfCfJ19v6hiQHkjyS5CtJzm7t57T1I2371NKULkkaxkKO6K8HDs9Y/zhwS1VtBJ4Ctrf27cBTVfVK4JbWT5I0JkMFfZJ1wFXAbW09wOXA3tZlD3B1W97a1mnbt7T+kqQxGPaI/lPAR4DftfULgaer6kRbPwasbctrgaMAbfszrb8kaQzmDfok7wCerKpDM5vn6FpDbJv5uDuSHExycHp6eqhiJUkLN8wR/ZuAdyZ5DPgygymbTwGrk5y8hMI64Im2fAxYD9C2/z7ws9kPWlW7qmpTVW1as2bNogYhSTq1eYO+qm6qqnVVNQVcC9xbVe8G7gPe1bptA+5sy/vaOm37vVX1nCN6SdKZsZjz6G8EbkhyhMEc/O7Wvhu4sLXfAOxcXImSpMVY0NUrq+qbwDfb8qPAZXP0+TVwzQhqkySNwIr8ZKwkrSRej36EvKyCpOVoRR/Re80bSSvBig56SVoJDHpJ6pxBL0mdM+glqXMGvSR1zqCXpM4Z9JLUOYNekjpn0EtS5wx6SeqcQS9JnTPoJalzBr0kdc6gl6TOGfSS1DmD/nnyWvaSJoVBL0mdM+glqXMGvSR1zqCXpM4Z9JLUOYNekjpn0EtS5wx6SeqcQS9JnTPoJalzBv0E8bILkp4Pg16SOmfQS1LnDHpJ6pxBL0mdM+glqXMG/TLlGTaSRsWgl6TOGfSS1Ll5gz7J+iT3JTmc5KEk17f2C5Lck+SRdnt+a0+SzyQ5kuSBJJcu9SAkSac2zBH9CeDDVfUaYDNwXZLXAjuB/VW1Edjf1gGuADa2rx3ArSOvWpI0tFXzdaiq48DxtvyLJIeBtcBW4C2t2x7gm8CNrf2LVVXA/UlWJ7m4PY7m4RuwkkZtQXP0SaaAS4ADwMtOhne7fWnrthY4OuNux1rb7MfakeRgkoPT09MLr1ySNJShgz7Ji4GvAR+sqp+fruscbfWchqpdVbWpqjatWbNm2DIkSQs0VNAneQGDkL+9qu5ozT9JcnHbfjHwZGs/Bqyfcfd1wBOjKVeStFDDnHUTYDdwuKo+OWPTPmBbW94G3Dmj/b3t7JvNwDPOz0vS+Mz7ZizwJuA9wPeSfLe1/TVwM/DVJNuBx4Fr2ra7gSuBI8CvgPeNtGJJ0oIMc9bNfzD3vDvAljn6F3DdIuuSJI2In4yVpM4Z9JLUOYNekjpn0J8hXnZY0rgY9JLUOYNekjpn0C+A0y+SJpFBL0mdM+glqXMGfYecYpI0k0EvSZ0z6BfJo2dJy51BL0mdM+glqXPDXI9+ojmtImml84hekjpn0EtS5wx6Sepc93P0o+A8v6RJ5hG9JHWuq6Cf1A8vTWrdkiZDV0G/GAatpF4Z9JLUuYkPeqc9JOn0Jj7oJUmnZ9B3zr94JBn0E84glzQfg16SOucnY2dZjkfHy6Wmk3U8dvNVY65E0kJ4RC9JnTPoJalzBr0kdc45+gm1VPP2o5iHdy5fWl48ol8hPA1TWrkMeknqnFM3mneqZdi/BPyLQVqePKLXoox6SsgpJmn0luSIPsnbgU8DZwG3VdXNS/E8p+Kbgac3tfMuHrv5qkUF6vO970rfNyt9/BqPkQd9krOAzwJ/BhwDvpVkX1V9f9TPpf9v1EfWsPhAWupgm+vxT/ecBq1WoqU4or8MOFJVjwIk+TKwFTDotSgzQ3oUv9SGDf2V+MthJY65Z0sR9GuBozPWjwF/sgTPowkyVzDPbhv2qHyhz7GQIF9oTcPedzHPP2xNs8e6mHENW/Nso3q9hjlBwF9Cw0tVjfYBk2uAt1XVX7b19wCXVdX7Z/XbAexoq68CHl7E014E/HQR918OehgD9DEOx7A8OIb5/UFVrZmv01Ic0R8D1s9YXwc8MbtTVe0Cdo3iCZMcrKpNo3iscelhDNDHOBzD8uAYRmcpTq/8FrAxyYYkZwPXAvuW4HkkSUMY+RF9VZ1I8lfAvzI4vfJzVfXQqJ9HkjScJTmPvqruBu5eisc+hZFMAY1ZD2OAPsbhGJYHxzAiI38zVpK0vHgJBEnq3MQHfZK3J3k4yZEkO8ddzzCSrE9yX5LDSR5Kcn1rvyDJPUkeabfnj7vW+SQ5K8l3kny9rW9IcqCN4SvtDfllK8nqJHuT/KDtjzdM2n5I8qH2ffRgki8leeEk7Ickn0vyZJIHZ7TN+dpn4DPt5/yBJJeOr/JnnWIMf9u+nx5I8k9JVs/YdlMbw8NJ3nam6pzooJ9xuYUrgNcCf57kteOtaigngA9X1WuAzcB1re6dwP6q2gjsb+vL3fXA4RnrHwduaWN4Ctg+lqqG92ngG1X1auD1DMYyMfshyVrgA8CmqnodgxMgrmUy9sMXgLfPajvVa38FsLF97QBuPUM1zucLPHcM9wCvq6o/Av4LuAmg/YxfC/xhu8/ftQxbchMd9My43EJV/QY4ebmFZa2qjlfVt9vyLxiEy1oGte9p3fYAV4+nwuEkWQdcBdzW1gNcDuxtXZb1GJK8BHgzsBugqn5TVU8zYfuBwUkVL0qyCjgXOM4E7Ieq+nfgZ7OaT/XabwW+WAP3A6uTXHxmKj21ucZQVf9WVSfa6v0MPksEgzF8uar+t6p+CBxhkGFLbtKDfq7LLawdUy3PS5Ip4BLgAPCyqjoOg18GwEvHV9lQPgV8BPhdW78QeHrGN/ly3x+vAKaBz7fpp9uSnMcE7Yeq+hHwCeBxBgH/DHCIydoPM53qtZ/Un/W/AP6lLY9tDJMe9JmjbWJOI0ryYuBrwAer6ufjrmchkrwDeLKqDs1snqPrct4fq4BLgVur6hLglyzjaZq5tDnsrcAG4OXAeQymOWZbzvthGJP2vUWSjzKYpr39ZNMc3c7IGCY96Ie63MJylOQFDEL+9qq6ozX/5OSfo+32yXHVN4Q3Ae9M8hiDKbPLGRzhr25TCLD898cx4FhVHWjrexkE/yTth7cCP6yq6ar6LXAH8EYmaz/MdKrXfqJ+1pNsA94BvLuePYd9bGOY9KCfyMsttLns3cDhqvrkjE37gG1teRtw55mubVhVdVNVrauqKQav+71V9W7gPuBdrdtyH8OPgaNJXtWatjC4nPbE7AcGUzabk5zbvq9OjmFi9sMsp3rt9wHvbWffbAaeOTnFs9xk8I+XbgTeWVW/mrFpH3BtknOSbGDwxvJ/npGiqmqiv4ArGbyz/d/AR8ddz5A1/ymDP9keAL7bvq5kMMe9H3ik3V4w7lqHHM9bgK+35Ve0b94jwD8C54y7vnlq/2PgYNsX/wycP2n7Afgb4AfAg8A/AOdMwn4AvsTgfYXfMjja3X6q157BtMdn28/59xicZbRcx3CEwVz8yZ/tv5/R/6NtDA8DV5ypOv1krCR1btKnbiRJ8zDoJalzBr0kdc6gl6TOGfSS1DmDXpI6Z9BLUucMeknq3P8BbUYaU0fWy8kAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import collections\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "content_length_count = collections.OrderedDict(content_length_count)\n",
    "plt.bar(range(len(content_length_count)), list(content_length_count.values()), align='center')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'content_length_count' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-20-509d87ae0db6>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mcontent_length_count\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'content_length_count' is not defined"
     ]
    }
   ],
   "source": [
    "content_length_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import jieba\n",
    "def stopwordset(stopword_path):\n",
    "  stopwords = [line.strip() for line in open(stopword_path,encoding='UTF-8').readlines()]\n",
    "  return set(stopwords)\n",
    "\n",
    "\n",
    "def seg_sentence(sentence, stopwordlist):\n",
    "  sentence_depart = jieba.cut(sentence.strip())\n",
    "  seg_result = [x.strip() for x in sentence_depart if x not in stopwordlist]\n",
    "  seg_result = [x for x in seg_result if x]\n",
    "  return seg_result\n",
    "\n",
    "def gen_data_set(data_path, label_path, stopword_path):\n",
    "  stopwords = stopwordset(stopword_path)\n",
    "  data_df = pd.read_csv(data_path)\n",
    "  label_df = pd.read_csv(label_path)\n",
    "  label_map = {}\n",
    "  for index, row in label_df.iterrows():\n",
    "    label_map[row['id']] = row['label']\n",
    "\n",
    "  results = []\n",
    "  for index, row in data_df.iterrows():\n",
    "    content = row['content']\n",
    "    if not content or type(content) == float:\n",
    "      continue\n",
    "    seg_result = seg_sentence(content, stopwords)\n",
    "    label = label_map[row['id']]\n",
    "    results.append([seg_result, label])\n",
    "  return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = gen_data_set('C:\\\\Users\\\\bai\\\\Downloads\\\\Train\\\\Train_DataSet.csv',  'C:\\\\Users\\\\bai\\\\Downloads\\\\Train\\\\Train_DataSet_Label.csv', 'C:\\\\Users\\\\bai\\\\Documents\\\\codebase\\\\DFSolution\\\\SentimentAnalysis\\\\stopwords.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['月',\n",
       "  '26',\n",
       "  '日',\n",
       "  '傍晚',\n",
       "  '点',\n",
       "  '30',\n",
       "  '分',\n",
       "  '大连',\n",
       "  '甘井子区',\n",
       "  '华北',\n",
       "  '路',\n",
       "  '周水子',\n",
       "  '立交桥',\n",
       "  '发生',\n",
       "  '意外事故',\n",
       "  '一位',\n",
       "  '行人',\n",
       "  '横穿',\n",
       "  '华北',\n",
       "  '路时',\n",
       "  '一辆',\n",
       "  '大客车',\n",
       "  '撞倒',\n",
       "  '事故现场',\n",
       "  '见到',\n",
       "  '有人',\n",
       "  '冒险',\n",
       "  '穿越',\n",
       "  '马路',\n",
       "  '网友',\n",
       "  '拍摄',\n",
       "  '现场',\n",
       "  '照片',\n",
       "  '事发',\n",
       "  '地点',\n",
       "  '位于',\n",
       "  '周水子',\n",
       "  '立交桥',\n",
       "  '沃尔玛',\n",
       "  '超市',\n",
       "  '华北',\n",
       "  '路',\n",
       "  '路段',\n",
       "  '一名',\n",
       "  '行人',\n",
       "  '躺倒',\n",
       "  '华北',\n",
       "  '路',\n",
       "  '伤情',\n",
       "  '不详',\n",
       "  '手中',\n",
       "  '提着',\n",
       "  '纸袋子',\n",
       "  '飞',\n",
       "  '数米',\n",
       "  '外',\n",
       "  '事故现场',\n",
       "  '马路',\n",
       "  '中央',\n",
       "  '人行横道',\n",
       "  '肇事',\n",
       "  '大客车',\n",
       "  '停',\n",
       "  '不远处',\n",
       "  '车头',\n",
       "  '撞击',\n",
       "  '破损',\n",
       "  '痕迹',\n",
       "  '目击者',\n",
       "  '称',\n",
       "  '撞',\n",
       "  '伤者',\n",
       "  '横穿马路',\n",
       "  '说法',\n",
       "  '未',\n",
       "  '交警部门',\n",
       "  '进一步',\n",
       "  '证实',\n",
       "  '居民',\n",
       "  '称',\n",
       "  '居民区',\n",
       "  '大型',\n",
       "  '商超',\n",
       "  '横穿马路',\n",
       "  '为数不少',\n",
       "  '事故现场',\n",
       "  '有人',\n",
       "  '车流',\n",
       "  '中',\n",
       "  '横穿马路',\n",
       "  '事故',\n",
       "  '原因',\n",
       "  '伤者',\n",
       "  '伤情',\n",
       "  '进一步',\n",
       "  '中'],\n",
       " 2]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results[299]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
